1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.solr.update.processor;
19
20 import org.apache.solr.common.SolrInputDocument;
21 import org.apache.solr.schema.IndexSchema;
22 import org.joda.time.DateTime;
23 import org.joda.time.DateTimeZone;
24 import org.joda.time.format.DateTimeFormat;
25 import org.joda.time.format.DateTimeFormatter;
26 import org.joda.time.format.ISODateTimeFormat;
27 import org.junit.BeforeClass;
28
29 import java.util.Date;
30 import java.util.HashMap;
31 import java.util.HashSet;
32 import java.util.LinkedHashMap;
33 import java.util.Map;
34 import java.util.Set;
35
36
37
38
39
40 public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
41 private static final double EPSILON = 1E-15;
42
43 @BeforeClass
44 public static void beforeClass() throws Exception {
45 initCore("solrconfig-parsing-update-processor-chains.xml", "schema12.xml");
46 }
47
48 public void testParseDateRoundTrip() throws Exception {
49 IndexSchema schema = h.getCore().getLatestSchema();
50 assertNotNull(schema.getFieldOrNull("date_dt"));
51 String dateString = "2010-11-12T13:14:15.168Z";
52 SolrInputDocument d = processAdd("parse-date", doc(f("id", "9"), f("date_dt", dateString)));
53 assertNotNull(d);
54 DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
55 DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
56 assertTrue(d.getFieldValue("date_dt") instanceof Date);
57 assertEquals(dateTime.getMillis(), ((Date) d.getFieldValue("date_dt")).getTime());
58 assertU(commit());
59 assertQ(req("id:9"), "//date[@name='date_dt'][.='" + dateString + "']");
60 }
61
62 public void testParseTrieDateRoundTrip() throws Exception {
63 IndexSchema schema = h.getCore().getLatestSchema();
64 assertNotNull(schema.getFieldOrNull("date_tdt"));
65 String dateString = "2010-11-12T13:14:15.168Z";
66 SolrInputDocument d = processAdd("parse-date", doc(f("id", "39"), f("date_tdt", dateString)));
67 assertNotNull(d);
68 DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
69 DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
70 assertTrue(d.getFieldValue("date_tdt") instanceof Date);
71 assertEquals(dateTime.getMillis(), ((Date) d.getFieldValue("date_tdt")).getTime());
72 assertU(commit());
73 assertQ(req("id:39"), "//date[@name='date_tdt'][.='" + dateString + "']");
74 }
75
76
77 public void testParseDateFieldNotInSchema() throws Exception {
78 IndexSchema schema = h.getCore().getLatestSchema();
79 assertNull(schema.getFieldOrNull("not_in_schema"));
80 String dateString = "2010-11-12T13:14:15.168Z";
81 DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
82 DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
83
84 SolrInputDocument d = processAdd("parse-date-no-run-processor",
85 doc(f("id", "18"), f("not_in_schema", dateString)));
86 assertNotNull(d);
87 assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
88 assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
89
90 d = processAdd("parse-date-no-run-processor",
91 doc(f("id", "36"), f("not_in_schema", "not a date", dateString)));
92 assertNotNull(d);
93 for (Object val : d.getFieldValues("not_in_schema")) {
94
95 assertTrue(val instanceof String);
96 }
97
98 d = processAdd("parse-date-no-run-processor",
99 doc(f("id", "72"), f("not_in_schema", dateString, "not a date")));
100 assertNotNull(d);
101 for (Object val : d.getFieldValues("not_in_schema")) {
102
103 assertTrue(val instanceof String);
104 }
105 }
106
107 public void testParseDateNonUTCdefaultTimeZoneRoundTrip() throws Exception {
108 IndexSchema schema = h.getCore().getLatestSchema();
109 assertNotNull(schema.getFieldOrNull("date_dt"));
110 String dateStringNoTimeZone = "2010-11-12T13:14:15.168";
111 String dateStringUTC = dateStringNoTimeZone + "Z";
112
113
114 String dateStringUSEasternTimeAsUTC = "2010-11-12T18:14:15.168Z";
115
116 SolrInputDocument d = processAdd
117 ("parse-date-non-UTC-defaultTimeZone", doc(f("id", "99"), f("dateUTC_dt", dateStringUTC),
118 f("dateNoTimeZone_dt", dateStringNoTimeZone)));
119 assertNotNull(d);
120 String pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
121 DateTimeFormatter dateTimeFormatterUTC = DateTimeFormat.forPattern(pattern);
122 DateTime dateTimeUTC = dateTimeFormatterUTC.parseDateTime(dateStringUTC);
123 assertTrue(d.getFieldValue("dateUTC_dt") instanceof Date);
124 assertTrue(d.getFieldValue("dateNoTimeZone_dt") instanceof Date);
125 assertEquals(dateTimeUTC.getMillis(), ((Date) d.getFieldValue("dateUTC_dt")).getTime());
126 assertU(commit());
127 assertQ(req("id:99")
128 ,"//date[@name='dateUTC_dt'][.='" + dateStringUTC + "']"
129 ,"//date[@name='dateNoTimeZone_dt'][.='" + dateStringUSEasternTimeAsUTC + "']");
130 }
131
132 public void testParseDateExplicitNotInSchemaSelector() throws Exception {
133 IndexSchema schema = h.getCore().getLatestSchema();
134 assertNull(schema.getFieldOrNull("not_in_schema"));
135 String dateString = "2010-11-12T13:14:15.168Z";
136 DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
137 DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
138
139 SolrInputDocument d = processAdd("parse-date-explicit-not-in-schema-selector-no-run-processor",
140 doc(f("id", "88"), f("not_in_schema", dateString)));
141 assertNotNull(d);
142 assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
143 assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
144 }
145
146 public void testParseDateExplicitTypeClassSelector() throws Exception {
147 IndexSchema schema = h.getCore().getLatestSchema();
148 assertNotNull(schema.getFieldOrNull("date_dt"));
149 String dateString = "2010-11-12T13:14:15.168Z";
150 DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
151 DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
152
153 SolrInputDocument d = processAdd("parse-date-explicit-typeclass-selector-no-run-processor",
154 doc(f("id", "77"), f("date_dt", dateString)));
155 assertNotNull(d);
156 assertTrue(d.getFieldValue("date_dt") instanceof Date);
157 assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("date_dt")).getTime());
158 }
159
160 public void testParseUSPacificDate() throws Exception {
161 IndexSchema schema = h.getCore().getLatestSchema();
162 assertNull(schema.getFieldOrNull("not_in_schema"));
163 String dateString = "8/9/2010";
164 String dateStringUTC = "2010-08-09T07:00:00.000Z";
165 SolrInputDocument d = processAdd("US-Pacific-parse-date-no-run-processor",
166 doc(f("id", "288"), f("not_in_schema", dateString)));
167 assertNotNull(d);
168 assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
169 assertEquals(dateStringUTC,
170 (new DateTime(((Date)d.getFieldValue("not_in_schema")).getTime(),DateTimeZone.UTC)).toString());
171 }
172
173 public void testParseDateFormats() throws Exception {
174 String[] formatExamples = {
175 "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "2010-01-15T00:00:00.000Z",
176 "yyyy-MM-dd'T'HH:mm:ss,SSSZ", "2010-01-15T00:00:00,000Z",
177 "yyyy-MM-dd'T'HH:mm:ss.SSS", "2010-01-15T00:00:00.000",
178 "yyyy-MM-dd'T'HH:mm:ss,SSS", "2010-01-15T00:00:00,000",
179 "yyyy-MM-dd'T'HH:mm:ssZ", "2010-01-15T00:00:00Z",
180 "yyyy-MM-dd'T'HH:mm:ss", "2010-01-15T00:00:00",
181 "yyyy-MM-dd'T'HH:mmZ", "2010-01-15T00:00Z",
182 "yyyy-MM-dd'T'HH:mm", "2010-01-15T00:00",
183 "yyyy-MM-dd HH:mm:ss.SSSZ", "2010-01-15 00:00:00.000Z",
184 "yyyy-MM-dd HH:mm:ss,SSSZ", "2010-01-15 00:00:00,000Z",
185 "yyyy-MM-dd HH:mm:ss.SSS", "2010-01-15 00:00:00.000",
186 "yyyy-MM-dd HH:mm:ss,SSS", "2010-01-15 00:00:00,000",
187 "yyyy-MM-dd HH:mm:ssZ", "2010-01-15 00:00:00Z",
188 "yyyy-MM-dd HH:mm:ss", "2010-01-15 00:00:00",
189 "yyyy-MM-dd HH:mmZ", "2010-01-15 00:00Z",
190 "yyyy-MM-dd HH:mm", "2010-01-15 00:00",
191 "yyyy-MM-dd hh:mm a", "2010-01-15 12:00 AM",
192 "yyyy-MM-dd hh:mma", "2010-01-15 12:00AM",
193 "yyyy-MM-dd", "2010-01-15",
194 "EEE MMM dd HH:mm:ss Z yyyy", "Fri Jan 15 00:00:00 +0000 2010",
195 "EEE MMM dd HH:mm:ss yyyy Z", "Fri Jan 15 00:00:00 2010 +00:00",
196 "EEE MMM dd HH:mm:ss yyyy", "Fri Jan 15 00:00:00 2010",
197 "EEE, dd MMM yyyy HH:mm:ss Z", "Fri, 15 Jan 2010 00:00:00 +00:00",
198 "EEEE, dd-MMM-yy HH:mm:ss Z", "Friday, 15-Jan-10 00:00:00 +00:00",
199 "EEEE, MMMM dd, yyyy", "Friday, January 15, 2010",
200 "MMMM dd, yyyy", "January 15, 2010",
201 "MMM. dd, yyyy", "Jan. 15, 2010"
202 };
203
204 IndexSchema schema = h.getCore().getLatestSchema();
205 assertNotNull(schema.getFieldOrNull("dateUTC_dt"));
206
207 String dateTimePattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
208 DateTimeFormatter dateTimeFormatterUTC = DateTimeFormat.forPattern(dateTimePattern);
209 DateTime dateTimeUTC = dateTimeFormatterUTC.parseDateTime(formatExamples[1]);
210
211 for (int i = 0 ; i < formatExamples.length ; i += 2) {
212 String format = formatExamples[i];
213 String dateString = formatExamples[i + 1];
214 String id = "95" + i;
215 SolrInputDocument d = processAdd("parse-date-UTC-defaultTimeZone-no-run-processor",
216 doc(f("id", id), f("dateUTC_dt", dateString)));
217 assertNotNull(d);
218 assertTrue("date '" + dateString + "' with format '" + format + "' is not mutated to a Date",
219 d.getFieldValue("dateUTC_dt") instanceof Date);
220 assertEquals("date '" + dateString + "' with format '" + format + "' mismatched milliseconds",
221 dateTimeUTC.getMillis(), ((Date)d.getFieldValue("dateUTC_dt")).getTime());
222 }
223 }
224
225 public void testParseFrenchDate() throws Exception {
226 IndexSchema schema = h.getCore().getLatestSchema();
227 assertNull(schema.getFieldOrNull("not_in_schema"));
228 String frenchDateString = "le vendredi 15 janvier 2010";
229 String dateString = "2010-01-15T00:00:00.000Z";
230 DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
231 DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
232 SolrInputDocument d = processAdd("parse-french-date-UTC-defaultTimeZone-no-run-processor",
233 doc(f("id", "88"), f("not_in_schema", frenchDateString)));
234 assertNotNull(d);
235 assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
236 assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
237 }
238
239 public void testFailedParseMixedDate() throws Exception {
240 IndexSchema schema = h.getCore().getLatestSchema();
241 assertNull(schema.getFieldOrNull("not_in_schema"));
242 DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
243 Map<Object,Object> mixed = new HashMap<>();
244 String[] dateStrings = { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
245 for (String dateString : dateStrings) {
246 mixed.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
247 }
248 Double extraDouble = 29.554d;
249 mixed.put(extraDouble, extraDouble);
250 SolrInputDocument d = processAdd("parse-date-no-run-processor",
251 doc(f("id", "7201"), f("not_in_schema", mixed.values())));
252 assertNotNull(d);
253 boolean foundDouble = false;
254 for (Object o : d.getFieldValues("not_in_schema")) {
255 if (extraDouble == o) {
256 foundDouble = true;
257 } else {
258 assertTrue(o instanceof String);
259 }
260 mixed.values().remove(o);
261 }
262 assertTrue(foundDouble);
263 assertTrue(mixed.isEmpty());
264 }
265
266 public void testParseIntRoundTrip() throws Exception {
267 IndexSchema schema = h.getCore().getLatestSchema();
268 assertNotNull(schema.getFieldOrNull("int1_i"));
269 assertNotNull(schema.getFieldOrNull("int2_i"));
270 int value = 1089883491;
271 String intString1 = "1089883491";
272 String intString2 = "1,089,883,491";
273 SolrInputDocument d = processAdd("parse-int",
274 doc(f("id", "113"), f("int1_i", intString1), f("int2_i", intString2)));
275 assertNotNull(d);
276 assertTrue(d.getFieldValue("int1_i") instanceof Integer);
277 assertEquals(value, ((Integer)d.getFieldValue("int1_i")).intValue());
278 assertTrue(d.getFieldValue("int2_i") instanceof Integer);
279 assertEquals(value, ((Integer)d.getFieldValue("int2_i")).intValue());
280
281 assertU(commit());
282 assertQ(req("id:113")
283 ,"//int[@name='int1_i'][.='" + value + "']"
284 ,"//int[@name='int2_i'][.='" + value + "']");
285 }
286
287 public void testParseIntNonRootLocale() throws Exception {
288 IndexSchema schema = h.getCore().getLatestSchema();
289 assertNotNull(schema.getFieldOrNull("int_i"));
290 assertNull(schema.getFieldOrNull("not_in_schema"));
291 int value = 1089883491;
292 String intString1 = "1089883491";
293 String intString2 = "1 089 883 491";
294 SolrInputDocument d = processAdd("parse-int-russian-no-run-processor",
295 doc(f("id", "113"), f("int_i", intString1), f("not_in_schema", intString2)));
296 assertNotNull(d);
297 assertTrue(d.getFieldValue("int_i") instanceof Integer);
298 assertEquals(value, ((Integer)d.getFieldValue("int_i")).intValue());
299 assertTrue(d.getFieldValue("not_in_schema") instanceof Integer);
300 assertEquals(value, ((Integer)d.getFieldValue("not_in_schema")).intValue());
301 }
302
303 public void testParseTrieIntRoundTrip() throws Exception {
304 IndexSchema schema = h.getCore().getLatestSchema();
305 assertNotNull(schema.getFieldOrNull("int1_ti"));
306 assertNotNull(schema.getFieldOrNull("int2_ti"));
307 int value = 1089883491;
308 String intString1 = "1089883491";
309 String intString2 = "1,089,883,491";
310 SolrInputDocument d = processAdd("parse-int",
311 doc(f("id", "113"), f("int1_ti", intString1), f("int2_ti", intString2)));
312 assertNotNull(d);
313 assertTrue(d.getFieldValue("int1_ti") instanceof Integer);
314 assertEquals(value, ((Integer)d.getFieldValue("int1_ti")).intValue());
315 assertTrue(d.getFieldValue("int2_ti") instanceof Integer);
316 assertEquals(value, ((Integer)d.getFieldValue("int2_ti")).intValue());
317
318 assertU(commit());
319 assertQ(req("id:113")
320 ,"//int[@name='int1_ti'][.='" + value + "']"
321 ,"//int[@name='int2_ti'][.='" + value + "']");
322 }
323
324 public void testIntOverflow() throws Exception {
325 IndexSchema schema = h.getCore().getLatestSchema();
326 assertNull(schema.getFieldOrNull("not_in_schema1"));
327 assertNull(schema.getFieldOrNull("not_in_schema2"));
328 long longValue1 = (long)Integer.MAX_VALUE + 100L;
329 long longValue2 = (long)Integer.MIN_VALUE - 100L;
330 String longString1 = Long.toString(longValue1);
331 String longString2 = Long.toString(longValue2);
332 SolrInputDocument d = processAdd("parse-int-no-run-processor",
333 doc(f("id", "282"), f("not_in_schema1", longString1), f("not_in_schema2", longString2)));
334 assertNotNull(d);
335 assertTrue(d.getFieldValue("not_in_schema1") instanceof String);
336 assertTrue(d.getFieldValue("not_in_schema2") instanceof String);
337 }
338
339 public void testFailedParseMixedInt() throws Exception {
340 IndexSchema schema = h.getCore().getLatestSchema();
341 assertNull(schema.getFieldOrNull("not_in_schema"));
342 Map<Object,Object> mixed = new HashMap<>();
343 Float floatVal = 294423.0f;
344 mixed.put(85, "85");
345 mixed.put(floatVal, floatVal);
346 mixed.put(-2894518, "-2,894,518");
347 mixed.put(1879472193, "1,879,472,193");
348 SolrInputDocument d = processAdd("parse-int-no-run-processor",
349 doc(f("id", "7202"), f("not_in_schema", mixed.values())));
350 assertNotNull(d);
351 boolean foundFloat = false;
352 for (Object o : d.getFieldValues("not_in_schema")) {
353 if (floatVal == o) {
354 foundFloat = true;
355 } else {
356 assertTrue(o instanceof String);
357 }
358 mixed.values().remove(o);
359 }
360 assertTrue(foundFloat);
361 assertTrue(mixed.isEmpty());
362 }
363
364 public void testParseLongRoundTrip() throws Exception {
365 IndexSchema schema = h.getCore().getLatestSchema();
366 assertNotNull(schema.getFieldOrNull("long1_l"));
367 assertNotNull(schema.getFieldOrNull("long2_l"));
368 long value = 1089883491L;
369 String longString1 = "1089883491";
370 String longString2 = "1,089,883,491";
371 SolrInputDocument d = processAdd("parse-long",
372 doc(f("id", "113"), f("long1_l", longString1), f("long2_l", longString2)));
373 assertNotNull(d);
374 assertTrue(d.getFieldValue("long1_l") instanceof Long);
375 assertEquals(value, ((Long) d.getFieldValue("long1_l")).longValue());
376 assertTrue(d.getFieldValue("long2_l") instanceof Long);
377 assertEquals(value, ((Long)d.getFieldValue("long2_l")).longValue());
378
379 assertU(commit());
380 assertQ(req("id:113")
381 ,"//long[@name='long1_l'][.='" + value + "']"
382 ,"//long[@name='long2_l'][.='" + value + "']");
383 }
384
385 public void testParseLongNonRootLocale() throws Exception {
386 IndexSchema schema = h.getCore().getLatestSchema();
387 assertNotNull(schema.getFieldOrNull("long_l"));
388 assertNull(schema.getFieldOrNull("not_in_schema"));
389 long value = 1089883491L;
390 String longString1 = "1089883491";
391 String longString2 = "1 089 883 491";
392 SolrInputDocument d = processAdd("parse-long-russian-no-run-processor",
393 doc(f("id", "113"), f("long_l", longString1), f("not_in_schema", longString2)));
394 assertNotNull(d);
395 assertTrue(d.getFieldValue("long_l") instanceof Long);
396 assertEquals(value, ((Long)d.getFieldValue("long_l")).longValue());
397 assertTrue(d.getFieldValue("not_in_schema") instanceof Long);
398 assertEquals(value, ((Long)d.getFieldValue("not_in_schema")).longValue());
399 }
400
401 public void testParseTrieLongRoundTrip() throws Exception {
402 IndexSchema schema = h.getCore().getLatestSchema();
403 assertNotNull(schema.getFieldOrNull("long1_tl"));
404 assertNotNull(schema.getFieldOrNull("long2_tl"));
405 long value = 1089883491L;
406 String longString1 = "1089883491";
407 String longString2 = "1,089,883,491";
408 SolrInputDocument d = processAdd("parse-long",
409 doc(f("id", "113"), f("long1_tl", longString1), f("long2_tl", longString2)));
410 assertNotNull(d);
411 assertTrue(d.getFieldValue("long1_tl") instanceof Long);
412 assertEquals(value, ((Long)d.getFieldValue("long1_tl")).longValue());
413 assertTrue(d.getFieldValue("long2_tl") instanceof Long);
414 assertEquals(value, ((Long)d.getFieldValue("long2_tl")).longValue());
415
416 assertU(commit());
417 assertQ(req("id:113")
418 ,"//long[@name='long1_tl'][.='" + value + "']"
419 ,"//long[@name='long2_tl'][.='" + value + "']");
420 }
421
422 public void testFailedParseMixedLong() throws Exception {
423 IndexSchema schema = h.getCore().getLatestSchema();
424 assertNull(schema.getFieldOrNull("not_in_schema"));
425 Map<Object,Object> mixed = new HashMap<>();
426 Float floatVal = 294423.0f;
427 mixed.put(85L, "85");
428 mixed.put(floatVal, floatVal);
429 mixed.put(-2894518L, "-2,894,518");
430 mixed.put(1879472193L, "1,879,472,193");
431 SolrInputDocument d = processAdd("parse-long-no-run-processor",
432 doc(f("id", "7204"), f("not_in_schema", mixed.values())));
433 assertNotNull(d);
434 boolean foundFloat = false;
435 for (Object o : d.getFieldValues("not_in_schema")) {
436 if (floatVal == o) {
437 foundFloat = true;
438 } else {
439 assertTrue(o instanceof String);
440 }
441 mixed.values().remove(o);
442 }
443 assertTrue(foundFloat);
444 assertTrue(mixed.isEmpty());
445 }
446
447 public void testParseFloatRoundTrip() throws Exception {
448 IndexSchema schema = h.getCore().getLatestSchema();
449 assertNotNull(schema.getFieldOrNull("float1_f"));
450 assertNotNull(schema.getFieldOrNull("float2_f"));
451 float value = 10898.83491f;
452 String floatString1 = "10898.83491";
453 String floatString2 = "10,898.83491";
454 SolrInputDocument d = processAdd("parse-float",
455 doc(f("id", "128"), f("float1_f", floatString1), f("float2_f", floatString2)));
456 assertNotNull(d);
457 assertTrue(d.getFieldValue("float1_f") instanceof Float);
458 assertEquals(value, (Float)d.getFieldValue("float1_f"), EPSILON);
459 assertTrue(d.getFieldValue("float2_f") instanceof Float);
460 assertEquals(value, (Float)d.getFieldValue("float2_f"), EPSILON);
461
462 assertU(commit());
463 assertQ(req("id:128")
464 ,"//float[@name='float1_f'][.='" + value + "']"
465 ,"//float[@name='float2_f'][.='" + value + "']");
466 }
467
468 public void testParseFloatNonRootLocale() throws Exception {
469 IndexSchema schema = h.getCore().getLatestSchema();
470 assertNotNull(schema.getFieldOrNull("float_f"));
471 assertNull(schema.getFieldOrNull("not_in_schema"));
472 float value = 10898.83491f;
473 String floatString1 = "10898,83491";
474 String floatString2 = "10 898,83491";
475 SolrInputDocument d = processAdd("parse-float-french-no-run-processor",
476 doc(f("id", "140"), f("float_f", floatString1),
477 f("not_in_schema", floatString2)));
478 assertNotNull(d);
479 assertTrue(d.getFieldValue("float_f") instanceof Float);
480 assertEquals(value, (Float)d.getFieldValue("float_f"), EPSILON);
481 assertTrue(d.getFieldValue("not_in_schema") instanceof Float);
482 assertEquals(value, (Float)d.getFieldValue("not_in_schema"), EPSILON);
483 }
484
485 public void testParseTrieFloatRoundTrip() throws Exception {
486 IndexSchema schema = h.getCore().getLatestSchema();
487 assertNotNull(schema.getFieldOrNull("float1_tf"));
488 assertNotNull(schema.getFieldOrNull("float2_tf"));
489 float value = 10898.83491f;
490 String floatString1 = "10898.83491";
491 String floatString2 = "10,898.83491";
492 SolrInputDocument d = processAdd("parse-float",
493 doc(f("id", "728"), f("float1_tf", floatString1), f("float2_tf", floatString2)));
494 assertNotNull(d);
495 assertTrue(d.getFieldValue("float1_tf") instanceof Float);
496 assertEquals(value, (Float)d.getFieldValue("float1_tf"), EPSILON);
497 assertTrue(d.getFieldValue("float2_tf") instanceof Float);
498 assertEquals(value, (Float)d.getFieldValue("float2_tf"), EPSILON);
499
500 assertU(commit());
501 assertQ(req("id:728")
502 ,"//float[@name='float1_tf'][.='" + value + "']"
503 ,"//float[@name='float2_tf'][.='" + value + "']");
504 }
505
506 public void testMixedFloats() throws Exception {
507 IndexSchema schema = h.getCore().getLatestSchema();
508 assertNotNull(schema.getFieldOrNull("float_tf"));
509 Map<Float,Object> mixedFloats = new HashMap<>();
510 mixedFloats.put(85.0f, "85");
511 mixedFloats.put(2894518.0f, "2,894,518");
512 mixedFloats.put(2.94423E-9f, 2.94423E-9f);
513 mixedFloats.put(48794721.937f, "48,794,721.937");
514 SolrInputDocument d = processAdd("parse-float-no-run-processor",
515 doc(f("id", "342"), f("float_tf", mixedFloats.values())));
516 assertNotNull(d);
517 for (Object o : d.getFieldValues("float_tf")) {
518 assertTrue(o instanceof Float);
519 mixedFloats.remove(o);
520 }
521 assertTrue(mixedFloats.isEmpty());
522 }
523
524 public void testFailedParseMixedFloat() throws Exception {
525 IndexSchema schema = h.getCore().getLatestSchema();
526 assertNull(schema.getFieldOrNull("not_in_schema"));
527 Map<Object,Object> mixed = new HashMap<>();
528 Long longVal = 294423L;
529 mixed.put(85L, "85");
530 mixed.put(longVal, longVal);
531 mixed.put(-2894518L, "-2,894,518");
532 mixed.put(1879472193L, "1,879,472,193");
533 SolrInputDocument d = processAdd("parse-float-no-run-processor",
534 doc(f("id", "7205"), f("not_in_schema", mixed.values())));
535 assertNotNull(d);
536 boolean foundLong = false;
537 for (Object o : d.getFieldValues("not_in_schema")) {
538 if (longVal == o) {
539 foundLong = true;
540 } else {
541 assertTrue(o instanceof String);
542 }
543 mixed.values().remove(o);
544 }
545 assertTrue(foundLong);
546 assertTrue(mixed.isEmpty());
547 }
548
549 public void testParseDoubleRoundTrip() throws Exception {
550 IndexSchema schema = h.getCore().getLatestSchema();
551 assertNotNull(schema.getFieldOrNull("double1_d"));
552 assertNotNull(schema.getFieldOrNull("double2_d"));
553 double value = 10898.83491;
554 String doubleString1 = "10898.83491";
555 String doubleString2 = "10,898.83491";
556 SolrInputDocument d = processAdd("parse-double",
557 doc(f("id", "128"), f("double1_d", doubleString1), f("double2_d", doubleString2)));
558 assertNotNull(d);
559 assertTrue(d.getFieldValue("double1_d") instanceof Double);
560 assertEquals(value, (Double)d.getFieldValue("double1_d"), EPSILON);
561 assertTrue(d.getFieldValue("double2_d") instanceof Double);
562 assertEquals(value, (Double)d.getFieldValue("double2_d"), EPSILON);
563
564 assertU(commit());
565 assertQ(req("id:128")
566 ,"//double[@name='double1_d'][.='" + value + "']"
567 ,"//double[@name='double2_d'][.='" + value + "']");
568 }
569
570 public void testParseDoubleNonRootLocale() throws Exception {
571 IndexSchema schema = h.getCore().getLatestSchema();
572 assertNotNull(schema.getFieldOrNull("double_d"));
573 assertNull(schema.getFieldOrNull("not_in_schema"));
574 double value = 10898.83491;
575 String doubleString1 = "10898,83491";
576 String doubleString2 = "10 898,83491";
577 SolrInputDocument d = processAdd("parse-double-french-no-run-processor",
578 doc(f("id", "140"), f("double_d", doubleString1),
579 f("not_in_schema", doubleString2)));
580 assertNotNull(d);
581 assertTrue(d.getFieldValue("double_d") instanceof Double);
582 assertEquals(value, (Double)d.getFieldValue("double_d"), EPSILON);
583 assertTrue(d.getFieldValue("not_in_schema") instanceof Double);
584 assertEquals(value, (Double)d.getFieldValue("not_in_schema"), EPSILON);
585 }
586
587 public void testParseTrieDoubleRoundTrip() throws Exception {
588 IndexSchema schema = h.getCore().getLatestSchema();
589 assertNotNull(schema.getFieldOrNull("double1_td"));
590 assertNotNull(schema.getFieldOrNull("double2_td"));
591 double value = 10898.83491;
592 String doubleString1 = "10898.83491";
593 String doubleString2 = "10,898.83491";
594 SolrInputDocument d = processAdd("parse-double",
595 doc(f("id", "728"), f("double1_td", doubleString1), f("double2_td", doubleString2)));
596 assertNotNull(d);
597 assertTrue(d.getFieldValue("double1_td") instanceof Double);
598 assertEquals(value, (Double)d.getFieldValue("double1_td"), EPSILON);
599 assertTrue(d.getFieldValue("double2_td") instanceof Double);
600 assertEquals(value, (Double)d.getFieldValue("double2_td"), EPSILON);
601
602 assertU(commit());
603 assertQ(req("id:728")
604 ,"//double[@name='double1_td'][.='" + value + "']"
605 ,"//double[@name='double2_td'][.='" + value + "']");
606 }
607
608 public void testFailedParseMixedDouble() throws Exception {
609 IndexSchema schema = h.getCore().getLatestSchema();
610 assertNull(schema.getFieldOrNull("not_in_schema"));
611 Map<Object,Object> mixed = new HashMap<>();
612 Long longVal = 294423L;
613 mixed.put(85, "85.0");
614 mixed.put(longVal, longVal);
615 mixed.put(-2894.518, "-2,894.518");
616 mixed.put(187947.2193, "187,947.2193");
617 SolrInputDocument d = processAdd("parse-double-no-run-processor",
618 doc(f("id", "7206"), f("not_in_schema", mixed.values())));
619 assertNotNull(d);
620 boolean foundLong = false;
621 for (Object o : d.getFieldValues("not_in_schema")) {
622 if (longVal == o) {
623 foundLong = true;
624 } else {
625 assertTrue(o instanceof String);
626 }
627 mixed.values().remove(o);
628 }
629 assertTrue(foundLong);
630 assertTrue(mixed.isEmpty());
631 }
632
633 public void testParseBooleanRoundTrip() throws Exception {
634 IndexSchema schema = h.getCore().getLatestSchema();
635 assertNotNull(schema.getFieldOrNull("boolean1_b"));
636 assertNotNull(schema.getFieldOrNull("boolean2_b"));
637 boolean value1 = true;
638 boolean value2 = false;
639 SolrInputDocument d = processAdd("parse-boolean",
640 doc(f("id", "141"), f("boolean1_b", value1), f("boolean2_b", value2)));
641 assertNotNull(d);
642 assertTrue(d.getFieldValue("boolean1_b") instanceof Boolean);
643 assertEquals(value1, d.getFieldValue("boolean1_b"));
644 assertTrue(d.getFieldValue("boolean2_b") instanceof Boolean);
645 assertEquals(value2, d.getFieldValue("boolean2_b"));
646
647 assertU(commit());
648 assertQ(req("id:141")
649 ,"//bool[@name='boolean1_b'][.='" + value1 + "']"
650 ,"//bool[@name='boolean2_b'][.='" + value2 + "']");
651 }
652
653 public void testParseAlternateValueBooleans() throws Exception {
654 IndexSchema schema = h.getCore().getLatestSchema();
655 assertNotNull(schema.getFieldOrNull("boolean1_b"));
656 assertNotNull(schema.getFieldOrNull("boolean2_b"));
657 assertNotNull(schema.getFieldOrNull("boolean3_b"));
658 assertNotNull(schema.getFieldOrNull("boolean4_b"));
659 assertNotNull(schema.getFieldOrNull("boolean5_b"));
660 assertNull(schema.getFieldOrNull("not_in_schema"));
661 boolean[] values = { true, true, true, false, false, false };
662 String[] stringValues = { "on", "yes", "True", "Off", "no", "FALSE" };
663 String[] fieldNames = { "boolean1_b", "boolean2_b", "boolean3_b", "boolean4_b", "boolean5_b", "not_in_schema" };
664 SolrInputDocument d = doc(f("id", "55"));
665 for (int i = 0 ; i < values.length ; ++i) {
666 d.addField(fieldNames[i], stringValues[i]);
667 }
668 d = processAdd("parse-boolean-alternate-values-no-run-processor", d);
669 assertNotNull(d);
670
671 for (int i = 0 ; i < values.length ; ++i) {
672 assertTrue(d.getFieldValue(fieldNames[i]) instanceof Boolean);
673 assertEquals(values[i], d.getFieldValue(fieldNames[i]));
674 }
675 }
676
677 public void testParseAlternateSingleValuesBooleans() throws Exception {
678 IndexSchema schema = h.getCore().getLatestSchema();
679 assertNotNull(schema.getFieldOrNull("boolean1_b"));
680 assertNotNull(schema.getFieldOrNull("boolean2_b"));
681 boolean[] values = { true, false };
682 String[] stringValues = { "yup", "nope" };
683 String[] fieldNames = { "boolean1_b", "boolean2_b" };
684 SolrInputDocument d = doc(f("id", "59"));
685 for (int i = 0 ; i < values.length ; ++i) {
686 d.addField(fieldNames[i], stringValues[i]);
687 }
688 d = processAdd("parse-boolean-alternate-single-values-no-run-processor", d);
689 assertNotNull(d);
690
691 for (int i = 0 ; i < values.length ; ++i) {
692 assertTrue(d.getFieldValue(fieldNames[i]) instanceof Boolean);
693 assertEquals(values[i], d.getFieldValue(fieldNames[i]));
694 }
695
696
697 stringValues = new String[] { "true", "false" };
698 d = doc(f("id", "593"));
699 for (int i = 0 ; i < values.length ; ++i) {
700 d.addField(fieldNames[i], stringValues[i]);
701 }
702 d = processAdd("parse-boolean-alternate-single-values-no-run-processor", d);
703 assertNotNull(d);
704
705 for (int i = 0 ; i < values.length ; ++i) {
706 assertTrue(d.getFieldValue(fieldNames[i]) instanceof String);
707 }
708 }
709
710 public void testFailedParseMixedBoolean() throws Exception {
711 IndexSchema schema = h.getCore().getLatestSchema();
712 assertNull(schema.getFieldOrNull("not_in_schema"));
713 Map<Object,Object> mixed = new HashMap<>();
714 Long longVal = 294423L;
715 mixed.put(true, "true");
716 mixed.put(longVal, longVal);
717 mixed.put(false, "false");
718 mixed.put(true, "true");
719 SolrInputDocument d = processAdd("parse-boolean-no-run-processor",
720 doc(f("id", "7207"), f("not_in_schema", mixed.values())));
721 assertNotNull(d);
722 boolean foundLong = false;
723 for (Object o : d.getFieldValues("not_in_schema")) {
724 if (longVal == o) {
725 foundLong = true;
726 } else {
727 assertTrue(o instanceof String);
728 }
729 mixed.values().remove(o);
730 }
731 assertTrue(foundLong);
732 assertTrue(mixed.isEmpty());
733 }
734
735 public void testCascadingParsers() throws Exception {
736 IndexSchema schema = h.getCore().getLatestSchema();
737 final String fieldName = "not_in_schema";
738 assertNull(schema.getFieldOrNull(fieldName));
739 SolrInputDocument d = null;
740 String chain = "cascading-parsers-no-run-processor";
741
742 Map<Boolean,String> booleans = new HashMap<>();
743 booleans.put(true, "truE");
744 booleans.put(false, "False");
745 d = processAdd(chain, doc(f("id", "341"), f(fieldName, booleans.values())));
746 assertNotNull(d);
747 for (Object o : d.getFieldValues(fieldName)) {
748 assertTrue(o instanceof Boolean);
749 booleans.remove(o);
750 }
751 assertTrue(booleans.isEmpty());
752
753 Map<Integer,String> ints = new HashMap<>();
754 ints.put(2, "2");
755 ints.put(50928, "50928");
756 ints.put(86942008, "86,942,008");
757 d = processAdd(chain, doc(f("id", "333"), f(fieldName, ints.values())));
758 assertNotNull(d);
759 for (Object o : d.getFieldValues(fieldName)) {
760 assertTrue(o instanceof Integer);
761 ints.remove(o);
762 }
763 assertTrue(ints.isEmpty());
764
765 Map<Long,String> longs = new HashMap<>();
766 longs.put(2L, "2");
767 longs.put(50928L, "50928");
768 longs.put(86942008987654L, "86,942,008,987,654");
769 d = processAdd(chain, doc(f("id", "342"), f(fieldName, longs.values())));
770 assertNotNull(d);
771 for (Object o : d.getFieldValues(fieldName)) {
772 assertTrue(o instanceof Long);
773 longs.remove(o);
774 }
775 assertTrue(longs.isEmpty());
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792 Map<Double,String> doubles = new HashMap<>();
793 doubles.put(2.0, "2.");
794 doubles.put(509.28, "509.28");
795 doubles.put(86942.008, "86,942.008");
796 d = processAdd(chain, doc(f("id", "342"), f(fieldName, doubles.values())));
797 assertNotNull(d);
798 for (Object o : d.getFieldValues(fieldName)) {
799 assertTrue(o instanceof Double);
800 longs.remove(o);
801 }
802
803 DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
804 Map<Date,String> dates = new HashMap<>();
805 String[] dateStrings = { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
806 for (String dateString : dateStrings) {
807 dates.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
808 }
809 d = processAdd(chain, doc(f("id", "343"), f(fieldName, dates.values())));
810 assertNotNull(d);
811 for (Object o : d.getFieldValues(fieldName)) {
812 assertTrue(o instanceof Date);
813 dates.remove(o);
814 }
815 assertTrue(dates.isEmpty());
816
817 Map<Double,String> mixedLongsAndDoubles = new LinkedHashMap<>();
818 mixedLongsAndDoubles.put(85.0, "85");
819 mixedLongsAndDoubles.put(2.94423E-9, "2.94423E-9");
820 mixedLongsAndDoubles.put(2894518.0, "2,894,518");
821 mixedLongsAndDoubles.put(48794721.937, "48,794,721.937");
822 d = processAdd(chain, doc(f("id", "344"), f(fieldName, mixedLongsAndDoubles.values())));
823 assertNotNull(d);
824 for (Object o : d.getFieldValues(fieldName)) {
825 assertTrue(o instanceof Double);
826 mixedLongsAndDoubles.remove(o);
827 }
828 assertTrue(mixedLongsAndDoubles.isEmpty());
829
830 Set<String> mixed = new HashSet<>();
831 mixed.add("true");
832 mixed.add("1682-07-22T18:33:00.000Z");
833 mixed.add("2,894,518");
834 mixed.add("308,393,131,379,900");
835 mixed.add("48,794,721.937");
836 d = processAdd(chain, doc(f("id", "345"), f(fieldName, mixed)));
837 assertNotNull(d);
838 for (Object o : d.getFieldValues(fieldName)) {
839 assertTrue(o instanceof String);
840 }
841
842 Map<Double,Object> mixedDoubles = new LinkedHashMap<>();
843 mixedDoubles.put(85.0, "85");
844 mixedDoubles.put(2.94423E-9, 2.94423E-9);
845 mixedDoubles.put(2894518.0, "2,894,518");
846 mixedDoubles.put(48794721.937, "48,794,721.937");
847 d = processAdd(chain, doc(f("id", "3391"), f(fieldName, mixedDoubles.values())));
848 assertNotNull(d);
849 for (Object o : d.getFieldValues(fieldName)) {
850 assertTrue(o instanceof Double);
851 mixedDoubles.remove(o);
852 }
853 assertTrue(mixedDoubles.isEmpty());
854
855 Map<Integer,Object> mixedInts = new LinkedHashMap<>();
856 mixedInts.put(85, "85");
857 mixedInts.put(294423, 294423);
858 mixedInts.put(-2894518, "-2,894,518");
859 mixedInts.put(1879472193, "1,879,472,193");
860 d = processAdd(chain, doc(f("id", "3392"), f(fieldName, mixedInts.values())));
861 assertNotNull(d);
862 for (Object o : d.getFieldValues(fieldName)) {
863 assertTrue(o instanceof Integer);
864 mixedInts.remove(o);
865 }
866 assertTrue(mixedInts.isEmpty());
867
868 Map<Long,Object> mixedLongs = new LinkedHashMap<>();
869 mixedLongs.put(85L, "85");
870 mixedLongs.put(42944233L, 42944233L);
871 mixedLongs.put(2894518L, "2,894,518");
872 mixedLongs.put(48794721937L, "48,794,721,937");
873 d = processAdd(chain, doc(f("id", "3393"), f(fieldName, mixedLongs.values())));
874 assertNotNull(d);
875 for (Object o : d.getFieldValues(fieldName)) {
876 assertTrue(o instanceof Long);
877 mixedLongs.remove(o);
878 }
879 assertTrue(mixedLongs.isEmpty());
880
881 Map<Boolean,Object> mixedBooleans = new LinkedHashMap<>();
882 mixedBooleans.put(true, "true");
883 mixedBooleans.put(false, false);
884 mixedBooleans.put(false, "false");
885 mixedBooleans.put(true, "true");
886 d = processAdd(chain, doc(f("id", "3394"), f(fieldName, mixedBooleans.values())));
887 assertNotNull(d);
888 for (Object o : d.getFieldValues(fieldName)) {
889 assertTrue(o instanceof Boolean);
890 mixedBooleans.remove(o);
891 }
892 assertTrue(mixedBooleans.isEmpty());
893
894 dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
895 Map<Date,Object> mixedDates = new HashMap<>();
896 dateStrings = new String[] { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
897 for (String dateString : dateStrings) {
898 mixedDates.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
899 }
900 Date extraDate = dateTimeFormatter.parseDateTime("2003-04-24").toDate();
901 mixedDates.put(extraDate, extraDate);
902 d = processAdd(chain, doc(f("id", "3395"), f(fieldName, mixedDates.values())));
903 assertNotNull(d);
904 for (Object o : d.getFieldValues(fieldName)) {
905 assertTrue(o instanceof Date);
906 mixedDates.remove(o);
907 }
908 assertTrue(mixedDates.isEmpty());
909 }
910 }